import requests
from lxml import etree
import time
import pymongo
from fake_useragent import UserAgent
ua = UserAgent()
headers = {'User-Agent':ua.random,
           'Connection':'keep-alive'}
client = pymongo.MongoClient('localhost',27017)
mydb = client['zhuanzhuan']
tongcheng_url = mydb['tongcheng_url']
tongcheng_info = mydb['tongcheng_info']

def get_links(channel,pages):
    list_view = '{}pn{}/'.format(channel,str(pages))
    try:
        html = requests.get(list_view,headers=headers)
        time.sleep(2)
        selector = etree.HTML(html.text)
        if selector.xpath('//tr'):
            infos = selector.xpath('//tr')
            for info in infos:
                url = info.xpath('td[2]/a/@href')[0]
                tongcheng_url.insert_one({'url':url})
            else:
                pass
        else:
            pass
    except requests.exceptions.ConnectionError:
        pass

def get_info(url):
    html = requests.get(url,headers = headers)
    selector = etree.HTML(html.text)
    try:
        content = selector.xpath('//td[@class=""t]/text()')[0]
        title = selector.xpath('//td[@class="t"]/a/text()')[0]
        address = selector.xpath('//td[@class="t"]/span[@class="f1"]/a/text()')[0].strip()
        time = selector.xpath('//td[@class="t"]/span[@class="f1"]/text()')[0].strip()
        price = selector.xpath('//td[@class="tc"]/text()')[0].strip()
        url = url
        data = {
            'title': title,
            'content': content,
            'address': address,
            'time': time,
            'price': price,
            'url': url
        }
        tongcheng_info.insert_one(data)
    except IndexError:
        pass


